
%% bare_conf.tex
%% V1.3
%% 2007/01/11
%% by Michael Shell
%% See:
%% http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.7 or later) with an IEEE conference paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/tex-archive/macros/latex/contrib/IEEEtran/
%% and
%% http://www.ieee.org/

%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE! 
%% User assumes all risk.
%% In no event shall IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including  **
%% ** renaming them and changing author support contact information. **
%%
%% File list of work: IEEEtran.cls, IEEEtran_HOWTO.pdf, bare_adv.tex,
%%                    bare_conf.tex, bare_jrnl.tex, bare_jrnl_compsoc.tex
%%*************************************************************************

% *** Authors should verify (and, if needed, correct) their LaTeX system  ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. IEEE's font choices can trigger bugs that do  ***
% *** not appear when using other class files.                            ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/



% Note that the a4paper option is mainly intended so that authors in
% countries using A4 can easily print to A4 and see how their papers will
% look in print - the typesetting of the document will not typically be
% affected with changes in paper size (but the bottom and side margins will).
% Use the testflow package mentioned above to verify correct handling of
% both paper sizes by the user's LaTeX system.
%
% Also note that the "draftcls" or "draftclsnofoot", not "draft", option
% should be used if it is desired that the figures are to be displayed in
% draft mode.
%
\documentclass[conference]{IEEEtran}
% Add the compsoc option for Computer Society conferences.
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[conference]{../sty/IEEEtran}





% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)


% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
%   % pdf code
% \else
%   % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/oberdiek/
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.






% *** CITATION PACKAGES ***
%
%\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 4.0 (2003-05-27) and later if using hyperref.sty. cite.sty does
% not currently provide for hyperlinked citations.
% The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/cite/
% The documentation is contained in the cite.sty file itself.






% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
   \usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
  %\graphicspath{{img/pdf/}{img/png/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
   \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
  % \usepackage[dvips]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../eps/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  % \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at: 
% http://www.ctan.org/tex-archive/macros/latex/required/graphics/
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found as epslatex.ps or
% epslatex.pdf at: http://www.ctan.org/tex-archive/info/
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex





% *** MATH PACKAGES ***
%
%\usepackage[cmex10]{amsmath}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics. If using
% it, be sure to load this package with the cmex10 option to ensure that
% only type 1 fonts will utilized at all point sizes. Without this option,
% it is possible that some math symbols, particularly those within
% footnotes, will be rendered in bitmap form which will result in a
% document that can not be IEEE Xplore compliant!
%
% Also, note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/





% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithms/
% There is also a support site at:
% http://algorithms.berlios.de/index.html
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithmicx/




% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/tools/


%\usepackage{mdwmath}
%\usepackage{mdwtab}
% Also highly recommended is Mark Wooding's extremely powerful MDW tools,
% especially mdwmath.sty and mdwtab.sty which are used to format equations
% and tables, respectively. The MDWtools set is already installed on most
% LaTeX systems. The lastest version and documentation is available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/mdwtools/


% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.


%\usepackage{eqparbox}
% Also of notable interest is Scott Pakin's eqparbox package for creating
% (automatically sized) equal width boxes - aka "natural width parboxes".
% Available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/eqparbox/





% *** SUBFIGURE PACKAGES ***
%\usepackage[tight,footnotesize]{subfigure}
% subfigure.sty was written by Steven Douglas Cochran. This package makes it
% easy to put subfigures in your figures. e.g., "Figure 1a and 1b". For IEEE
% work, it is a good idea to load it with the tight package option to reduce
% the amount of white space around the subfigures. subfigure.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at:
% http://www.ctan.org/tex-archive/obsolete/macros/latex/contrib/subfigure/
% subfigure.sty has been superceeded by subfig.sty.



%\usepackage[caption=false]{caption}
%\usepackage[font=footnotesize]{subfig}
% subfig.sty, also written by Steven Douglas Cochran, is the modern
% replacement for subfigure.sty. However, subfig.sty requires and
% automatically loads Axel Sommerfeldt's caption.sty which will override
% IEEEtran.cls handling of captions and this will result in nonIEEE style
% figure/table captions. To prevent this problem, be sure and preload
% caption.sty with its "caption=false" package option. This is will preserve
% IEEEtran.cls handing of captions. Version 1.3 (2005/06/28) and later 
% (recommended due to many improvements over 1.2) of subfig.sty supports
% the caption=false option directly:
%\usepackage[caption=false,font=footnotesize]{subfig}
%
% The latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/subfig/
% The latest version and documentation of caption.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/caption/




% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure. The latest version and documentation can be found at:
% http://www.ctan.org/tex-archive/macros/latex/base/



%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/sttools/
% Documentation is contained in the stfloats.sty comments as well as in the
% presfull.pdf file. Do not use the stfloats baselinefloat ability as IEEE
% does not allow \baselineskip to stretch. Authors submitting work to the
% IEEE should note that IEEE rarely uses double column equations and
% that authors should try to avoid such use. Do not be tempted to use the
% cuted.sty or midfloat.sty packages (also by Sigitas Tolusis) as IEEE does
% not format its papers in such ways.





% *** PDF, URL AND HYPERLINK PACKAGES ***
%
%\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/misc/
% Read the url.sty source comments for usage information. Basically,
% \url{my_url_here}.





% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex).         ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )


% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}


\begin{document}
%
% paper title
% can use linebreaks \\ within to get better formatting as desired
\title{From Graphical Output to Source Code at Different Abstraction Levels} %UI Inspection at Different Abstraction Levels


% author names and affiliations
% use a multiple column layout for up to three different
% affiliations
\author{\IEEEauthorblockN{Agon Bexheti, Shaban Shabani, Salman Mirghasemi, Claude Petitpierre}
\IEEEauthorblockA{\'Ecole Polytechnique F\'ed\'erale de Lausanne (EPFL) \\
Lausanne, Switzerland\\
Email: \{agon.bexheti, shaban.shabani, salman.mirghasemi, claude.petitpierre\}@epfl.ch}
}

% conference papers do not typically use \thanks and this command
% is locked out in conference mode. If really needed, such as for
% the acknowledgment of grants, issue a \IEEEoverridecommandlockouts
% after \documentclass

% for over three affiliations, or if they all won't fit within the width
% of the page, use this alternative format:
% 
%\author{\IEEEauthorblockN{Michael Shell\IEEEauthorrefmark{1},
%Homer Simpson\IEEEauthorrefmark{2},
%James Kirk\IEEEauthorrefmark{3}, 
%Montgomery Scott\IEEEauthorrefmark{3} and
%Eldon Tyrell\IEEEauthorrefmark{4}}
%\IEEEauthorblockA{\IEEEauthorrefmark{1}School of Electrical and Computer Engineering\\
%Georgia Institute of Technology,
%Atlanta, Georgia 30332--0250\\ Email: see http://www.michaelshell.org/contact.html}
%\IEEEauthorblockA{\IEEEauthorrefmark{2}Twentieth Century Fox, Springfield, USA\\
%Email: homer@thesimpsons.com}
%\IEEEauthorblockA{\IEEEauthorrefmark{3}Starfleet Academy, San Francisco, California 96678-2391\\
%Telephone: (800) 555--1212, Fax: (888) 555--1212}
%\IEEEauthorblockA{\IEEEauthorrefmark{4}Tyrell Inc., 123 Replicant Street, Los Angeles, California 90210--4321}}






% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}




% make the title area
\maketitle


\begin{abstract}
%\boldmath
Identifying the relevant source code to the task at hand is a frequent development activity. In cases that the code base is large or the developer is not familiar enough with the source code, performing this activity becomes a challenging job. A common approach used by developers is searching the  source code by related keywords to the task. This approach is difficult, because the developer has to guess and retry different keywords, and time-consuming, because the developer has to filter out the irrelevant items in the search result.

On the other hand, developers usually understand and check the program's functionality by looking at the graphical output, in particular the user interface, state changes. Therefore, it is very common that the developer starts the search for the relevant source code based on what has been observed in the graphical output. In this paper, we discuss the cases that developers need to identify the source code from the graphical output. 

A solution to the above problem can be achieved by providing a tool that directly identifies the source code that has a particular effect on the output. We investigated this idea by developing {\it Java UI Inspector (JUI)}, which lets developers to identify the relevant source code at different abstraction levels.



%Every software developer has experienced buggy output either in the GUI or the log file. The developer has to search and inspect the code to locate the related code which causes or writes the buggy output. This task is tedious and time consuming, particularly if the developer is not familiar enough with the code. The research question arises here: Is it possible to automatically locate the code from the buggy symptom? The developer just selects the buggy output on the screen and then gets the corresponding code without any additional effort. In this project we will pursue a novel practical approach for attacking this problem in general and specific (e.g., browsers) environments. 

\end{abstract}
% IEEEtran.cls defaults to using nonbold math in the Abstract.
% This preserves the distinction between vectors and scalars. However,
% if the conference you are submitting to favors bold math in the abstract,
% then you can use LaTeX's standard command \boldmath at the very start
% of the abstract to achieve this. Many IEEE journals/conferences frown on
% math in the abstract anyway.

% no keywords




% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle

\section{Introduction}


The size and complexity of programs that developers have to deal with is constantly increasing. Developers have to understand, test, and debug large programs which consist of numerous components and include thousands lines of code. These programs, in order to perform a task, may be dependent to several libraries, communicate with external programs, or run on several machines that make them even harder to be managed by developers. 


A common activity that appears almost in every development task is identifying the relevant source code to the task at hand. In cases that the code base is large or the developer is not familiar enough with the source code, performing this activity becomes a challenging job. A common approach used by developers is searching the  source code by related keywords to the task. This approach is difficult, because the developer has to guess and retry different keywords, and time-consuming, because the developer has to filter out the irrelevant items in the search result.


On the other hand, developers usually understand and check the program's functionality by looking at the graphical output, in particular the user interface, state changes. Therefore, it is very common that the developer starts the search for the relevant source code based on what has been observed in the graphical output. In this paper, we focus on this problem, i.e., identifying the relevant source code from the graphical output. 

\subsection{Motivation}
The ability to identify source code from graphical output can assist a developer in various kind of development tasks. Here, we enumerate three common cases as follows. 
%The first case is when the developer observes a bug symptom (e.g., a button with a wrong color or label) in the user interface. The scond case is when developers need to set graphical breakpoints (e.g., when a button is cliced but nothing happens). The last case is when developers need to locate a specific feature (e.g., search) and want to find the relevant source code.

\subsubsection{From Bug Symptoms to Source Code}
Debugging is often carried out by inspecting the buggy execution. A developer needs to start inspecting the execution from a point. Of course, the developer can start from the beginning of the buggy execution, but it may not be the most efficient way of selecting the first point. A reasonable candidate point to start inspection is the place where the bug symptoms appear. Some bug symptoms such as exceptions or error messages usually provide data (e.g.,  the error line number) that helps the developer to identify the execution point. For example, the developer can set a breakpoint on the exception to halt the execution whenever the exception occurs. 

Unfortunately, selecting the buggy point where bug symptoms appear is not always a straightforward task. For other types of bug symptoms, such as wrong output values, issues in the graphical user interface, execution slow down, and so on, developers may have a hard time locating the corresponding buggy point. They have to guess the relevant code or conditions to the bug symptoms and then search over the buggy execution. As we discuss in section~\ref{sec:discussion}, this problem appears in almost every backwards debugging approach. 

\subsubsection{Graphical Breakpoints}
In inspecting the buggy execution, a developer may need to understand how a particular event is handled. Suppose a buggy application with a button that does not properly function. The developer needs to halt the execution when the button is clicked and follow the execution trace. One approach is setting a breakpoint on the function that handles the event. Three issues appear here; First, the developer may not know the corresponding handler function(s). Second, the function may handle a broad range of events while the developer is only interested in events which are fired from a specific button. Third, later the developer may change the function name or location and therefore the breakpoint will not be anymore valid. 

Graphical breakpoints are developed to address the above issues in debugging web pages\cite{Barton2010}. A developer can set a graphical breakpoint on a UI element, e.g., a button, instead of the event handler function. Implementing graphical breakpoints is dependent to  identifying the UI element, the object(s) behind these elements and the event handlers. In web pages, most UI elements have identifiers and mapping them to corresponding objects and event handlers can be obtained through the document object model. However, in other more general UI models the answer to this problem can be challenging.

\subsubsection{Feature Location}
A larger category of development activites related to the identifying relevant source code problem is feature location. It is common that a developer looks for the source code related to a specific feature in the program in order to change or improve the feature. On the other hand, one tail of most features somehow appears in the user interface. Therefore, the UI is an appropriate gate to identifying the relevant source code to a feature. For example, suppose an application with a feature which lets the user to search in records. A user can search by entering the search constraints and pushing the ``search'' button. Knowing where the search form elments are created and added to the user interface, and how the search event is handled, can lead the developer to other related parts of source code.

\subsection{Challenges}
A few challenges must be addressed in order to build tools that support developers in the above problem. As we mentioned earlier, identifying the UI elements is an issue. This issue becomes a challenge considering that developers need to inspect the graphical output at different abstraction levels. For example a developer who wirtes a new component (e.g., a button) is interested in low-level graphical output such as lines and circles at the pixel level, whereas a developer who uses this component does not care about the low-level graphical output but wants to know where the component is created and added to the user interface. The latter developer may also be interested in to know how the component handles events.

%TODO
%Each one of cases we mentioned as the motivation to attacking the above problem, require different settings. 

\subsection{Contributions}
We develped a tool, {\it Java UI Inspector (JUI)}, which allows a developer to inspect graphical output of a Java program and identify the relevant source code at different abstraction levels. Morover, we demonstrate that the tool is practical and is efficient to be used for even very large applications.

\section{Introductory Example}
//A good example helps the reader to better understand our contributions. 

\section{JUI Implementation}
JUI consists of two inspection modes: postmortem and live. Both inspection modes provide algorithms that support
interactive debugging. Our decision to only trace the output interface of the target program is due to the fact that tracing everything is very costly and complex. In postmortem inspection mode, the program output is captured and analyzed after the program has stopped while in live mode, as the name implies, the developer has access to information about components as the target program is executing.

\subsection{Postmortem}
Postmortem inspection concentrates on instrumenting the following Java classes: awt.Graphics and awt.Graphics2D. For those who don't know, instrumentation in software engineering refers to the ability to diagnose errors, to write trace information and to measure the level of a product’s performance. Tracing is done on each method call of the Java graphics classes and enough information is absorbed as to be able to redraw the exact graphical output. The redrawing of the graphical output is displayed in a special viewer after the target program execution has ended. As a next step, while in the viewer, the developer can provide an input to JUI by specifying the interesting part of the screen (possibly the buggy output) and the source code that drew the output will be shown. 

\begin{figure}[h]
 \centering
 \includegraphics{img/img1}
 \caption{Steps for the Postmortem Inspection Mode}
 \label{fig1}
\end{figure}
 

Postmortem inspection implementation is done using Aspect Oriented Programming (AOP), specifically with AspectJ. AspectJ makes it possible to setup pointcuts for capturing calls to all of the methods in the Java graphics classes. Then the target program class files are weaved using the defined pointcut declarations. It is necessary that also some Java system classes to be weaved, which is explained in the end of this section.
For each component of the target program user interface, the graphics context is known. Thus, each single component can be redrawn and there is a mapping between UI components and graphics context.

Java painting mechanism is a single threaded painting model i.e all screen updates are done from a single thread. This is also known as Event dispatching thread (EDT). The event dispatching thread is the only valid and safe way to update the visual state of the user interface components. Trying to implement painting in Java using multiple threads can lead the user interface to an inconsistent state and very often this is the source of many common bugs in Java programs [reference here].

Recall that at this point the aim is to explore the graphics setting for each UI component. Knowing the fact that Java paintign mechanism is single threaded, it is sufficient to achieve this goal. First, it is necessary to capture the event when a component will start to be painted. This can be achieved with AspectJ by setting a before() pointcut that captures calls to the paint method of the component class and store in a variable the component that is going to be painted. In a similar fashion, creating a pointcut for each single method in Java graphics classes, events when these methods are invoked can be captured. At this poin the following data can be stored for each graphics method call: the method name (ex. drawLine()), the values of the parameters (ex. x1=5, y1=6, x2=10, y2=11), the source code line where the method was called (ex. MyPainting.class:34) and the graphics object itself (ex. instance of Graphics.class). The reason why the graphics object is needed is that few painting effects can be read directly from it such as painting color, stroke etc. All this information can be stored in a custom object serving as a data structure, ex. InstrumentedData.java. Once a call to the method paint is captured, all the upcoming captured calls to the methods of the graphics class are used to paint the same component for which paint was invoked. As a consecuence, a mapping between UI components and populated instances of InstrumentedData.java can be created. The key value to this map is the component and the instance of the InstrumentedData.java data structure is the map value. //TABLE 1 goes here

The stored data goes through a post-processing step. Most of the graphics methods have parameters that specify the area where the drawing will be made. These parameters are defined in the coordinate system of the graphics object that is used to do the painting. On the other hand, the graphics object first point (the point with coordinates x=0 and y=0) is equal with the parent container of the component which is being painted [reference here]. As it was explained above, for each graphics method we also store the values of these parameters as well. So the post-processing step updates these values relative to the screen.

Recall that the aim of postmortem inspection mode is to provide the source code line that produced a specific output. Once each component is redrawn (the output of the target program is constructed) the developer can select a part of this output and request the source code line that was used to draw the selected output. For each instrumented method the coordinates of the drawing and source code line where the method is called are known. Therefore, a mapping between pixels and source code lines is created. When the developer selects a part of the screen, this request is translated into a set of pixels the map is searched for the corresponding source code line. Since several pixels will be mapped with the same source code line only the distinct results are displayed.

Weaving Java system class files on load time is not an easy task. In principle, the load time weaver will weave all the classes that it sees during the class loading process and of course if the loaded class is within the scope of the weaver's configuration. The load time weaving mechanism of has a restriction when it comes to weaving java.* and javax.* classes and subclasses. Even if the restriction is removed it is still an issue to weave these classes due to the fact that the weaver is attached to the classloader after these classes are loaded. For this reason, we decided to pre-weave some of the necessary Java system classes and inform the Java virtual machine to use the weaved set of the system classes. There might be compatibility issues when using a pre-weaved set of Java system classes when trying the tool on a machine that has a different installed Java version from the one pre-weaved version of the pre-weaved classes. In future versions this issue can be solved by implementing a custom Java classloader and making it possible to weave on load time Java system classes.
\subsection{Live}
Using the live inspection mode the developer is able to make an inspection on UI components, i.e check properties of components such as component type, size, location, container etc. Implementation has been done using AspectJ and therefore the target program source code is not altered.

\begin{figure}[h]
 \centering
 \includegraphics{img/img2}
 \caption{Steps for the Live Inspection Mode}
 \label{fig1}
\end{figure}

The tool looks after events happening in the target program. Initially, capturing component construction is necessary and this is achieved using AspectJ pointcuts for capturing constructor calls on UI components and storing a copy of the component. As an example consider the event when when a JButton is created, then its copy is stored in a variable. Having already stored a copy of one of the UI components and using recursive function calls it is possible to create a tree of the entire component set of a frame. This step can be repeated for each available frame in the target program. To start inspecting the active frame, a glass pane component is added on top of the frame. Using the transparency property of the glass pane the developer does not lose the visible contact with the target program and still be able to do some inspection. The developer specifies the component that wants to inspect by clicking on the component itself. Actually, the click is on the glass pane and the algorithm makes sure that the click is translated to selecting the correct component. Using the UI component's location and size a map between pixels and UI components is created. As a result when the developer clicks on a component, the actual click is on top of the glass pane and the clicked coordinate pixels are searched in the map for the corresponding component. The tool is interactive and the inspection can be enabled and disabled at any time using a check box which appears in a separate frame. 

Live inspection provides the source code line of the event when a component is added to a parent component, for instance when a button is added to a panel. This is helpful for developers in situations of buggy output. Using live inspection fixing output bugs is fast and efficient as presented by cases in the evaluation section.
\section{Evaluation}
The tool has been tried in several test case applications using the load time weaving mechanism of the target program class files. It is necessary for the tool to use pre-weaved Java system classes which were weaved once on tool compile time. One of the testing metrics of the performance evaluation tests was the execution time overhead caused by the AspectJ weaver. For the postmortem inspection mode, we separately measure the execution overhead in the recording stage and the time it takes to replay what has been recorded. Simulating real case scenarios on the introductory example application and on more complex applications, from a human perspective there was no difference in speed during the recording stage. For a moderate simulation where the number of method calls to the graphics class is 30 000 in average, the execution overhead caused by instrumenting each graphics method call is approximately 2.3x. Furthermore replaying takes the same time as the duration of the effective recording (not considering the time when painting mechanism was idle). For instance, 10 seconds of painting with the introductory exampel application takes approximately 10 seconds to replay the traced output. Compared to most approaches of related work JUI faster in the sense that it causes less overhead in the recording stage and less time to load the recorded data.
\section{Discussion}
\label{sec:discussion}
Most debugging approaches allow the developer to move backwards on the buggy execution are dependent on starting from a buggy point. For example, Agrawal et al. mention this issue where they introduce their debugger based on dynamic slicing techniques which allow the developer to move backwards and find cuases of data or control flow problems \cite{Agrawal1993}; ``It requires that the programmer translate the externally visible symptoms of the fault into an internal program symptom in terms of a data or a control problem before the techniques described here may be used. This translation may not always be an easy one to make.'' Similar issue appears in other backwards debugging approaches as well as a recent approach, Querypoint debugging, which is based on reexecution in order to answer the developer's queries \cite{Mirghasemi2011}. 

% other types of output
% other potential improvements: recognizing the objects affect the output by dynamic analysis
% limitations , behavioral bug symptoms 

% We can also notify the developer if a paint method is called from a different thread than EDT.

\section{Related Work}

%Some of the new debugging approaches which are based on backwards 
% The problem "from bug symptoms to code" has been introduced
% Agrawal et al. 
% Querypoint

% Graphical Breakpoints
% Whyline
% Output
% SwingExplorer


% An example of a floating figure using the graphicx package.
% Note that \label must occur AFTER (or within) \caption.
% For figures, \caption should occur after the \includegraphics.
% Note that IEEEtran v1.7 and later has special internal code that
% is designed to preserve the operation of \label within \caption
% even when the captionsoff option is in effect. However, because
% of issues like this, it may be the safest practice to put all your
% \label just after \caption rather than within \caption{}.
%
% Reminder: the "draftcls" or "draftclsnofoot", not "draft", class
% option should be used if it is desired that the figures are to be
% displayed while in draft mode.
%
%\begin{figure}[!t]
%\centering
%\includegraphics[width=2.5in]{myfigure}
% where an .eps filename suffix will be assumed under latex, 
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
%\caption{Simulation Results}
%\label{fig_sim}
%\end{figure}

% Note that IEEE typically puts floats only at the top, even when this
% results in a large percentage of a column being occupied by floats.


% An example of a double column floating figure using two subfigures.
% (The subfig.sty package must be loaded for this to work.)
% The subfigure \label commands are set within each subfloat command, the
% \label for the overall figure must come after \caption.
% \hfil must be used as a separator to get equal spacing.
% The subfigure.sty package works much the same way, except \subfigure is
% used instead of \subfloat.
%
%\begin{figure*}[!t]
%\centerline{\subfloat[Case I]\includegraphics[width=2.5in]{subfigcase1}%
%\label{fig_first_case}}
%\hfil
%\subfloat[Case II]{\includegraphics[width=2.5in]{subfigcase2}%
%\label{fig_second_case}}}
%\caption{Simulation results}
%\label{fig_sim}
%\end{figure*}
%
% Note that often IEEE papers with subfigures do not employ subfigure
% captions (using the optional argument to \subfloat), but instead will
% reference/describe all of them (a), (b), etc., within the main caption.


% An example of a floating table. Note that, for IEEE style tables, the 
% \caption command should come BEFORE the table. Table text will default to
% \footnotesize as IEEE normally uses this smaller font for tables.
% The \label must come after \caption as always.
%
%\begin{table}[!t]
%% increase table row spacing, adjust to taste
%\renewcommand{\arraystretch}{1.3}
% if using array.sty, it might be a good idea to tweak the value of
% \extrarowheight as needed to properly center the text within the cells
%\caption{An Example of a Table}
%\label{table_example}
%\centering
%% Some packages, such as MDW tools, offer better commands for making tables
%% than the plain LaTeX2e tabular which is used here.
%\begin{tabular}{|c||c|}
%\hline
%One & Two\\
%\hline
%Three & Four\\
%\hline
%\end{tabular}
%\end{table}


% Note that IEEE does not put floats in the very first column - or typically
% anywhere on the first page for that matter. Also, in-text middle ("here")
% positioning is not used. Most IEEE journals/conferences use top floats
% exclusively. Note that, LaTeX2e, unlike IEEE journals/conferences, places
% footnotes above bottom floats. This can be corrected via the \fnbelowfloat
% command of the stfloats package.



\section{Conclusion}
The conclusion goes here.




% conference papers do not normally have an appendix


% use section* for acknowledgement
\section*{Acknowledgment}


The authors would like to thank...



% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}

% references section

% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://www.ctan.org/tex-archive/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
%\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{IEEEabrv,../bib/paper}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
\begin{thebibliography}{1}

%\bibitem{IEEEhowto:kopka}
%H.~Kopka and P.~W. Daly, \emph{A Guide to \LaTeX}, 3rd~ed.\hskip 1em plus
 % 0.5em minus 0.4em\relax Harlow, England: Addison-Wesley, 1999.

\bibitem{Agrawal1993}
H.~Agrawal, R.~A. Demillo, and E.~H. Spafford, ``Debugging with Dynamic Slicing and Backtracking,'' 
\emph{Software Practice and Experience}, vol. 23, pp. 589-616, 1993.

	
\bibitem{Barton2010}
J.~J. Barton, and J. Odvarko, ``Dynamic and graphical web page breakpoints,''
 in \emph{Proceedings of the 19th international
conference on World wide web (WWW)}, April, 2010. 

\bibitem{Horwitz2010}
S.~Horwitz, B.~Liblit, and M.~Polishchuk, ``Better Debugging via Output Tracing and Callstack-Sensitive Slicing,'' 
\emph{IEEE Transactions on Software Engineering (TSE)}, vol. 36, n. 1, pp. 7-19, 2010.

\bibitem{Ko2008}
A.~J. Ko, and B.~A. Myers. ``Debugging reinvented:
asking and answering why and why not questions
about program behavior,'' in \emph{Proceedings of the 30th international
conference on Software engineering (ICSE)}, May, 2008. 

\bibitem{Mirghasemi2011}
S.~Mirghasemi,  J.~J. Barton, and C.~Petitpierre, ``Querypoint: moving backwards on wrong values in the buggy execution,'' 
in \emph{Proceedings of the 19th ACM SIGSOFT Symposium on the Foundations
of Software Engineering and 13rd European Software Engineering Conference (ESEC/FSE)}, September, 2011. 
\end{thebibliography}

% that's all folks
\end{document}


